{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'Hello, TensorFlow!'\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "hello = tf.constant('Hello, TensorFlow!')\n",
    "sess = tf.Session()\n",
    "print(sess.run(hello))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "a = tf.constant(2, tf.int16)\n",
    "b = tf.constant(4, tf.float32)\n",
    "c = tf.constant(8, tf.float32)\n",
    "\n",
    "d = tf.Variable(2, tf.int16)\n",
    "e = tf.Variable(4, tf.float32)\n",
    "f = tf.Variable(8, tf.float32)\n",
    "\n",
    "g = tf.constant(np.zeros(shape=(2,2), dtype=np.float32)) #可以正常声明变量\n",
    "\n",
    "h = tf.zeros([11], tf.int16)\n",
    "i = tf.ones([2,2], tf.float32)\n",
    "j = tf.zeros([1000,4,3], tf.float64)\n",
    "\n",
    "k = tf.Variable(tf.zeros([2,2], tf.float32))\n",
    "l = tf.Variable(tf.zeros([5,6,5], tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.Variable 'Variable_2:0' shape=() dtype=int32_ref>\n",
      "8\n",
      "[[ 0.  0.]\n",
      " [ 0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "a = tf.constant(2, tf.int16)\n",
    "b = tf.constant(4, tf.float32)\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    a = tf.Variable(8, tf.float32)\n",
    "    b = tf.Variable(tf.zeros([2,2], tf.float32))\n",
    "    \n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print(f)\n",
    "    print(session.run(a))\n",
    "    print(session.run(b))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"add:0\", shape=(2,), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "a=tf.constant([1,2],name=\"a\")\n",
    "b=tf.constant([2,4],name=\"b\")\n",
    "result = a+b\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 4 6 8]\n",
      "[2 4 6 8]\n"
     ]
    }
   ],
   "source": [
    "a=tf.constant([1,2,3,4])\n",
    "b=tf.constant([1,2,3,4])\n",
    "result=a+b\n",
    "sess=tf.Session()\n",
    "print(sess.run(result))\n",
    "sess.close\n",
    "\n",
    "#输出 [2 4 6 8]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    a=tf.constant([1,2,3,4])\n",
    "    b=tf.constant([1,2,3,4])\n",
    "    result=a+b\n",
    "    print(sess.run(result))\n",
    "    \n",
    "#输出 [2 4 6 8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.11131823  2.38459873]]\n",
      "[[-0.11131823  2.38459873]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method BaseSession.close of <tensorflow.python.client.session.Session object at 0x000001D9DB209D30>>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1=tf.Variable(tf.random_normal([1,2],stddev=1,seed=1))\n",
    "\n",
    "#因为需要重复输入x，而每建一个x就会生成一个结点，计算图的效率会低。所以使用占位符\n",
    "x=tf.placeholder(tf.float32,shape=(1,2))\n",
    "x1=tf.constant([[0.7,0.9]])\n",
    "\n",
    "a=x+w1\n",
    "b=x1+w1\n",
    "\n",
    "sess=tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "#运行y时将占位符填上，feed_dict为字典，变量名不可变\n",
    "y_1=sess.run(a,feed_dict={x:[[0.7,0.9]]})\n",
    "y_2=sess.run(b)\n",
    "print(y_1)\n",
    "print(y_2)\n",
    "sess.close"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the distance between [[1 2]] and [[15 16]] -> [19.79899]\n",
      "the distance between [[3 4]] and [[13 14]] -> [14.142136]\n",
      "the distance between [[5 6]] and [[11 12]] -> [8.485281]\n",
      "the distance between [[7 8]] and [[ 9 10]] -> [2.8284271]\n"
     ]
    }
   ],
   "source": [
    "list_of_points1_ = [[1,2], [3,4], [5,6], [7,8]]\n",
    "list_of_points2_ = [[15,16], [13,14], [11,12], [9,10]]\n",
    "list_of_points1 = np.array([np.array(elem).reshape(1,2) for elem in list_of_points1_])\n",
    "list_of_points2 = np.array([np.array(elem).reshape(1,2) for elem in list_of_points2_])\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():   \n",
    "    \n",
    "    #我们使用 tf.placeholder() 创建占位符 ，在 session.run() 过程中再投递数据 \n",
    "    point1 = tf.placeholder(tf.float32, shape=(1, 2))\n",
    "    point2 = tf.placeholder(tf.float32, shape=(1, 2))\n",
    "    \n",
    "    def calculate_eucledian_distance(point1, point2):\n",
    "        difference = tf.subtract(point1, point2)\n",
    "        power2 = tf.pow(difference, tf.constant(2.0, shape=(1,2)))\n",
    "        add = tf.reduce_sum(power2)\n",
    "        eucledian_distance = tf.sqrt(add)\n",
    "        return eucledian_distance\n",
    "    \n",
    "    dist = calculate_eucledian_distance(point1, point2)\n",
    "    \n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()   \n",
    "    for ii in range(len(list_of_points1)):\n",
    "        point1_ = list_of_points1[ii]\n",
    "        point2_ = list_of_points2[ii]\n",
    "        \n",
    "        #使用feed_dict将数据投入到[dist]中\n",
    "        feed_dict = {point1 : point1_, point2 : point2_}\n",
    "        distance = session.run([dist], feed_dict=feed_dict)\n",
    "        print(\"the distance between {} and {} -> {}\".format(point1_, point2_, distance))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.81131822  1.48459876  0.06532937]\n",
      " [-2.4427042   0.0992484   0.59122431]]\n",
      "[[-0.81131822]\n",
      " [ 1.48459876]\n",
      " [ 0.06532937]]\n",
      "在迭代 0 次后，训练损失为 0.308504\n",
      "在迭代 1000 次后，训练损失为 0.0393406\n",
      "在迭代 2000 次后，训练损失为 0.0182158\n",
      "在迭代 3000 次后，训练损失为 0.0104779\n",
      "在迭代 4000 次后，训练损失为 0.00680374\n",
      "在迭代 5000 次后，训练损失为 0.00446512\n",
      "在迭代 6000 次后，训练损失为 0.00296797\n",
      "在迭代 7000 次后，训练损失为 0.00218553\n",
      "在迭代 8000 次后，训练损失为 0.00179452\n",
      "在迭代 9000 次后，训练损失为 0.0013211\n",
      "在迭代 10000 次后，训练损失为 0.000957699\n",
      "在迭代 11000 次后，训练损失为 0.00081103\n",
      "在迭代 12000 次后，训练损失为 0.000643147\n",
      "在迭代 13000 次后，训练损失为 0.00047439\n",
      "在迭代 14000 次后，训练损失为 0.00030086\n",
      "在迭代 15000 次后，训练损失为 0.000137936\n",
      "[[-0.81131822  3.84255528  3.38165283]\n",
      " [-2.4427042   1.98635983  3.50722313]]\n",
      "[[-0.81131822]\n",
      " [ 4.02907705]\n",
      " [ 2.60285187]]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from numpy.random import RandomState\n",
    "\n",
    "batch_size=10\n",
    "w1=tf.Variable(tf.random_normal([2,3],stddev=1,seed=1))\n",
    "w2=tf.Variable(tf.random_normal([3,1],stddev=1,seed=1))\n",
    "\n",
    "\n",
    "# None 可以根据batch 大小确定维度，在shape的一个维度上使用None，方便不大的batch\n",
    "x=tf.placeholder(tf.float32,shape=(None,2))\n",
    "y=tf.placeholder(tf.float32,shape=(None,1))\n",
    "\n",
    "a=tf.nn.relu(tf.matmul(x,w1))\n",
    "yhat=tf.nn.relu(tf.matmul(a,w2))\n",
    "\n",
    "#定义交叉熵为损失函数，训练过程使用Adam算法最小化交叉熵\n",
    "cross_entropy=-tf.reduce_mean(y*tf.log(tf.clip_by_value(yhat,1e-10,1.0)))\n",
    "train_step=tf.train.AdamOptimizer(0.001).minimize(cross_entropy)\n",
    "\n",
    "rdm=RandomState(1)\n",
    "data_size=512\n",
    "\n",
    "#生成两个特征，共data_size个样本\n",
    "X=rdm.rand(data_size,2)\n",
    "#定义规则给出样本标签，所有x1+x2<1的样本认为是正样本，其他为负样本。Y，1为正样本\n",
    "Y = [[int(x1+x2 < 1)] for (x1, x2) in X]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print(sess.run(w1))\n",
    "    print(sess.run(w2))\n",
    "    steps=15001\n",
    "    for i in range(steps):\n",
    "        \n",
    "        #选定每一个批量读取的首尾位置，确保在1个epoch内采样训练\n",
    "        start = i * batch_size % data_size\n",
    "        end = min(start + batch_size,data_size)\n",
    "        sess.run(train_step,feed_dict={x:X[start:end],y:Y[start:end]})\n",
    "        if i % 1000 == 0:\n",
    "            training_loss= sess.run(cross_entropy,feed_dict={x:X,y:Y})\n",
    "            print(\"在迭代 %d 次后，训练损失为 %g\"%(i,training_loss))\n",
    "        if i == steps-1:\n",
    "            print(sess.run(w1))\n",
    "            print(sess.run(w2))#输出更新后的权重矩阵"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上面的代码定义了一个简单的三层全连接网络（输入层、隐藏层和输出层分别为2、3和2个神经元），隐藏层和输出层的激活函数使用的是ReLU函数。该模型训练的样本总数为512，每次迭代读取的批量为10。这个简单的全连接网络以交叉熵为损失函数，并使用Adam优化算法进行权重更新。\n",
    "\n",
    "其中需要注意的几个函数如tf.nn.relu()代表调用ReLU激活函数，tf.matmul()为矩阵乘法等。tf.clip_by_value(yhat,1e-10,1.0)这一语句代表的是截断yhat的值，因为这一语句是嵌套在tf.log()函数内的，所以我们需要确保yhat的取值不会导致对数无穷大。\n",
    "\n",
    "tf.train.AdamOptimizer(learning_rate).minimize(cost_function)是进行训练的函数，其中我们采用的是Adam优化算法更新权重，并且需要提供学习速率和损失函数这两个参数。后面就是生成训练数据，X=rdm.rand(512,2)表示随机生成512个样本，每个样本有两个特征值。最后就是迭代运行了，这里我们计算出每一次迭代抽取数据的起始位置（start）和结束位置（end），并且每一次抽取的数据量为前面我们定义的批量，如果一个epoch最后剩余的数据少于批量大小，那就只是用剩余的数据进行训练。最后两句代码是为了计算训练损失并迭代一些次数后输出训练损失。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "import json\n",
    "import os\n",
    "\n",
    "#定义一些预处理函数\n",
    "\n",
    "def flatten_tf_array(array):\n",
    "    shape = array.get_shape().as_list()\n",
    "    return tf.reshape(array, [shape[0], shape[1] * shape[2] * shape[3]])\n",
    "\n",
    "def accuracy(predictions, labels):\n",
    "    return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) / predictions.shape[0])\n",
    "\n",
    "def randomize(dataset, labels):\n",
    "    permutation = np.random.permutation(labels.shape[0])\n",
    "    shuffled_dataset = dataset[permutation, :, :]\n",
    "    shuffled_labels = labels[permutation]\n",
    "    return shuffled_dataset, shuffled_labels\n",
    "\n",
    "def one_hot_encode(np_array):\n",
    "    return (np.arange(10) == np_array[:,None]).astype(np.float32)\n",
    "\n",
    "def reformat_data(dataset, labels, image_width, image_height, image_depth):\n",
    "    np_dataset_ = np.array([np.array(image_data).reshape(image_width, image_height, image_depth) for image_data in dataset])\n",
    "    np_labels_ = one_hot_encode(np.array(labels, dtype=np.float32))\n",
    "    np_dataset, np_labels = randomize(np_dataset_, np_labels_)\n",
    "    return np_dataset, np_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练集包含以下标签: [0 1 2 3 4 5 6 7 8 9]\n",
      "训练集维度 (50000, 32, 32, 3) (50000, 10)\n",
      "测试集维度 (10000, 32, 32, 3) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "cifar10_folder = './data/cifar10/'\n",
    "train_datasets = ['data_batch_1', 'data_batch_2', 'data_batch_3', 'data_batch_4', 'data_batch_5', ]\n",
    "test_dataset = ['test_batch']\n",
    "c10_image_height = 32\n",
    "c10_image_width = 32\n",
    "c10_image_depth = 3\n",
    "c10_num_labels = 10\n",
    "c10_image_size = 32 #Ahmet Taspinar的代码缺少了这一语句\n",
    "\n",
    "with open(cifar10_folder + test_dataset[0], 'rb') as f0:\n",
    "    c10_test_dict = pickle.load(f0, encoding='bytes')\n",
    "\n",
    "c10_test_dataset, c10_test_labels = c10_test_dict[b'data'], c10_test_dict[b'labels']\n",
    "test_dataset_cifar10, test_labels_cifar10 = reformat_data(c10_test_dataset, c10_test_labels, c10_image_size, c10_image_size, c10_image_depth)\n",
    "\n",
    "c10_train_dataset, c10_train_labels = [], []\n",
    "for train_dataset in train_datasets:\n",
    "    with open(cifar10_folder + train_dataset, 'rb') as f0:\n",
    "        c10_train_dict = pickle.load(f0, encoding='bytes')\n",
    "        c10_train_dataset_, c10_train_labels_ = c10_train_dict[b'data'], c10_train_dict[b'labels']\n",
    " \n",
    "        c10_train_dataset.append(c10_train_dataset_)\n",
    "        c10_train_labels += c10_train_labels_\n",
    "\n",
    "c10_train_dataset = np.concatenate(c10_train_dataset, axis=0)\n",
    "train_dataset_cifar10, train_labels_cifar10 = reformat_data(c10_train_dataset, c10_train_labels, c10_image_size, c10_image_size, c10_image_depth)\n",
    "del c10_train_dataset\n",
    "del c10_train_labels\n",
    "\n",
    "print(\"训练集包含以下标签: {}\".format(np.unique(c10_train_dict[b'labels'])))\n",
    "print('训练集维度', train_dataset_cifar10.shape, train_labels_cifar10.shape)\n",
    "print('测试集维度', test_dataset_cifar10.shape, test_labels_cifar10.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data/MNIST/train-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/train-labels-idx1-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"./data/MNIST/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data size:  55000\n",
      "Validating data size:  5000\n",
      "Testing data size:  10000\n"
     ]
    }
   ],
   "source": [
    "print(\"Training data size: \", mnist.train.num_examples) \n",
    "print (\"Validating data size: \", mnist.validation.num_examples) \n",
    "print (\"Testing data size: \", mnist.test.num_examples) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.38039219  0.37647063\n",
      "  0.3019608   0.46274513  0.2392157   0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.35294119  0.5411765\n",
      "  0.92156869  0.92156869  0.92156869  0.92156869  0.92156869  0.92156869\n",
      "  0.98431379  0.98431379  0.97254908  0.99607849  0.96078438  0.92156869\n",
      "  0.74509805  0.08235294  0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.\n",
      "  0.54901963  0.98431379  0.99607849  0.99607849  0.99607849  0.99607849\n",
      "  0.99607849  0.99607849  0.99607849  0.99607849  0.99607849  0.99607849\n",
      "  0.99607849  0.99607849  0.99607849  0.99607849  0.74117649  0.09019608\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.88627458  0.99607849  0.81568635\n",
      "  0.78039223  0.78039223  0.78039223  0.78039223  0.54509807  0.2392157\n",
      "  0.2392157   0.2392157   0.2392157   0.2392157   0.50196081  0.8705883\n",
      "  0.99607849  0.99607849  0.74117649  0.08235294  0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.\n",
      "  0.14901961  0.32156864  0.0509804   0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.13333334  0.83529419  0.99607849  0.99607849  0.45098042  0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.32941177  0.99607849  0.99607849  0.91764712  0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.32941177  0.99607849  0.99607849  0.91764712  0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.41568631  0.6156863   0.99607849  0.99607849  0.95294124  0.20000002\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.09803922  0.45882356  0.89411771\n",
      "  0.89411771  0.89411771  0.99215692  0.99607849  0.99607849  0.99607849\n",
      "  0.99607849  0.94117653  0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.26666668  0.4666667   0.86274517\n",
      "  0.99607849  0.99607849  0.99607849  0.99607849  0.99607849  0.99607849\n",
      "  0.99607849  0.99607849  0.99607849  0.55686277  0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.14509805  0.73333335  0.99215692\n",
      "  0.99607849  0.99607849  0.99607849  0.87450987  0.80784321  0.80784321\n",
      "  0.29411766  0.26666668  0.84313732  0.99607849  0.99607849  0.45882356\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.44313729\n",
      "  0.8588236   0.99607849  0.94901967  0.89019614  0.45098042  0.34901962\n",
      "  0.12156864  0.          0.          0.          0.          0.7843138\n",
      "  0.99607849  0.9450981   0.16078432  0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.66274512  0.99607849  0.6901961   0.24313727  0.          0.\n",
      "  0.          0.          0.          0.          0.          0.18823531\n",
      "  0.90588242  0.99607849  0.91764712  0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.07058824  0.48627454  0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.\n",
      "  0.32941177  0.99607849  0.99607849  0.65098041  0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.54509807  0.99607849  0.9333334   0.22352943  0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.\n",
      "  0.82352948  0.98039222  0.99607849  0.65882355  0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.94901967  0.99607849  0.93725497  0.22352943  0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.\n",
      "  0.34901962  0.98431379  0.9450981   0.33725491  0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.\n",
      "  0.01960784  0.80784321  0.96470594  0.6156863   0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.01568628  0.45882356  0.27058825  0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.        ]\n",
      "[ 0.  0.  0.  0.  0.  0.  0.  1.  0.  0.]\n",
      "[ 0.  0.  0.  0.  0.  0.  0.  1.  0.  0.]\n",
      "[ 0.  0.  0.  0.  0.  1.  0.  0.  0.  0.]\n"
     ]
    }
   ],
   "source": [
    "print(mnist.train.images[0]) \n",
    "print(mnist.train.labels[0]) \n",
    "print(mnist.test.labels[0]) \n",
    "print(mnist.validation.labels[0]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data/MNIST/train-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/train-labels-idx1-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"./data/MNIST/\", one_hot=True)\n",
    "\n",
    "\n",
    "INPUT_NODE = 784     \n",
    "OUTPUT_NODE = 10     \n",
    "LAYER1_NODE = 500         \n",
    "                              \n",
    "BATCH_SIZE = 100        \n",
    "\n",
    "# 模型相关的参数\n",
    "LEARNING_RATE_BASE = 0.8      \n",
    "LEARNING_RATE_DECAY = 0.99    \n",
    "REGULARAZTION_RATE = 0.0001   \n",
    "TRAINING_STEPS = 10000        \n",
    "MOVING_AVERAGE_DECAY = 0.99 \n",
    "\n",
    "def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):\n",
    "    # 使用滑动平均类\n",
    "    if avg_class == None:\n",
    "        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)\n",
    "        return tf.matmul(layer1, weights2) + biases2\n",
    "\n",
    "    else:\n",
    "        \n",
    "        layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))\n",
    "        return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)  \n",
    "    \n",
    "def train(mnist):\n",
    "    x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')\n",
    "    y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')\n",
    "    # 生成隐藏层的参数。\n",
    "    weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))\n",
    "    biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))\n",
    "    # 生成输出层的参数。\n",
    "    weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))\n",
    "    biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))\n",
    "\n",
    "    # 计算不含滑动平均类的前向传播结果\n",
    "    y = inference(x, None, weights1, biases1, weights2, biases2)\n",
    "    \n",
    "    # 定义训练轮数及相关的滑动平均类 \n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)\n",
    "    variables_averages_op = variable_averages.apply(tf.trainable_variables())\n",
    "    average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)\n",
    "    \n",
    "    # 计算交叉熵及其平均值\n",
    "    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))\n",
    "    cross_entropy_mean = tf.reduce_mean(cross_entropy)\n",
    "    \n",
    "    # 损失函数的计算\n",
    "    regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)\n",
    "    regularaztion = regularizer(weights1) + regularizer(weights2)\n",
    "    loss = cross_entropy_mean + regularaztion\n",
    "    \n",
    "    # 设置指数衰减的学习率。\n",
    "    learning_rate = tf.train.exponential_decay(\n",
    "        LEARNING_RATE_BASE,\n",
    "        global_step,\n",
    "        mnist.train.num_examples / BATCH_SIZE,\n",
    "        LEARNING_RATE_DECAY,\n",
    "        staircase=True)\n",
    "    \n",
    "    # 优化损失函数\n",
    "    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "    \n",
    "    # 反向传播更新参数和更新每一个参数的滑动平均值\n",
    "    with tf.control_dependencies([train_step, variables_averages_op]):\n",
    "        train_op = tf.no_op(name='train')\n",
    "\n",
    "    # 计算正确率\n",
    "    correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    # 初始化回话并开始训练过程。\n",
    "    with tf.Session() as sess:\n",
    "        tf.global_variables_initializer().run()\n",
    "        validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}\n",
    "        test_feed = {x: mnist.test.images, y_: mnist.test.labels} \n",
    "        \n",
    "        # 循环的训练神经网络。\n",
    "        for i in range(TRAINING_STEPS):\n",
    "            if i % 1000 == 0:\n",
    "                validate_acc = sess.run(accuracy, feed_dict=validate_feed)\n",
    "                print(\"After %d training step(s), validation accuracy using average model is %g \" % (i, validate_acc))\n",
    "            \n",
    "            xs,ys=mnist.train.next_batch(BATCH_SIZE)\n",
    "            sess.run(train_op,feed_dict={x:xs,y_:ys})\n",
    "\n",
    "        test_acc=sess.run(accuracy,feed_dict=test_feed)\n",
    "        print((\"After %d training step(s), test accuracy using average model is %g\" %(TRAINING_STEPS, test_acc)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After 0 training step(s), validation accuracy using average model is 0.0912 \n",
      "After 1000 training step(s), validation accuracy using average model is 0.9756 \n",
      "After 2000 training step(s), validation accuracy using average model is 0.9794 \n",
      "After 3000 training step(s), validation accuracy using average model is 0.9818 \n",
      "After 4000 training step(s), validation accuracy using average model is 0.9814 \n",
      "After 5000 training step(s), test accuracy using average model is 0.9834\n"
     ]
    }
   ],
   "source": [
    "train(mnist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After 0 training step(s), validation accuracy using average model is 0.1678 \n",
      "After 1000 training step(s), validation accuracy using average model is 0.9766 \n",
      "After 2000 training step(s), validation accuracy using average model is 0.981 \n",
      "After 3000 training step(s), validation accuracy using average model is 0.9808 \n",
      "After 4000 training step(s), validation accuracy using average model is 0.9824 \n",
      "After 5000 training step(s), validation accuracy using average model is 0.9822 \n",
      "After 6000 training step(s), validation accuracy using average model is 0.983 \n",
      "After 7000 training step(s), validation accuracy using average model is 0.9828 \n",
      "After 8000 training step(s), validation accuracy using average model is 0.9842 \n",
      "After 9000 training step(s), validation accuracy using average model is 0.9834 \n",
      "After 10000 training step(s), test accuracy using average model is 0.9851\n"
     ]
    }
   ],
   "source": [
    "avg_class = None\n",
    "train(mnist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized with learning_rate 0.001\n",
      "step 0000 : loss is 057.10, accuracy on training set 6.25 %, accuracy on test set 9.34 %\n",
      "step 1000 : loss is 001.63, accuracy on training set 62.50 %, accuracy on test set 34.26 %\n",
      "step 2000 : loss is 001.38, accuracy on training set 50.00 %, accuracy on test set 40.97 %\n",
      "step 3000 : loss is 001.78, accuracy on training set 31.25 %, accuracy on test set 41.43 %\n",
      "step 4000 : loss is 001.41, accuracy on training set 56.25 %, accuracy on test set 42.54 %\n",
      "step 5000 : loss is 001.52, accuracy on training set 37.50 %, accuracy on test set 45.20 %\n",
      "step 6000 : loss is 001.28, accuracy on training set 56.25 %, accuracy on test set 42.72 %\n",
      "step 7000 : loss is 001.29, accuracy on training set 56.25 %, accuracy on test set 46.89 %\n",
      "step 8000 : loss is 001.43, accuracy on training set 43.75 %, accuracy on test set 47.97 %\n",
      "step 9000 : loss is 001.58, accuracy on training set 31.25 %, accuracy on test set 49.00 %\n",
      "step 10000 : loss is 001.41, accuracy on training set 43.75 %, accuracy on test set 49.80 %\n"
     ]
    }
   ],
   "source": [
    "LENET5_LIKE_BATCH_SIZE = 32\n",
    "LENET5_LIKE_FILTER_SIZE = 5\n",
    "LENET5_LIKE_FILTER_DEPTH = 16\n",
    "LENET5_LIKE_NUM_HIDDEN = 120\n",
    "\n",
    "def variables_lenet5_like(filter_size = LENET5_LIKE_FILTER_SIZE, \n",
    "                          filter_depth = LENET5_LIKE_FILTER_DEPTH, \n",
    "                          num_hidden = LENET5_LIKE_NUM_HIDDEN,\n",
    "                          image_width = 32, image_height = 32, image_depth = 3, num_labels = 10):\n",
    "    \n",
    "    w1 = tf.Variable(tf.truncated_normal([filter_size, filter_size, image_depth, filter_depth], stddev=0.1))\n",
    "    b1 = tf.Variable(tf.zeros([filter_depth]))\n",
    "\n",
    "    w2 = tf.Variable(tf.truncated_normal([filter_size, filter_size, filter_depth, filter_depth], stddev=0.1))\n",
    "    b2 = tf.Variable(tf.constant(1.0, shape=[filter_depth]))\n",
    "   \n",
    "    w3 = tf.Variable(tf.truncated_normal([(image_width // 4)*(image_height // 4)*filter_depth , num_hidden], stddev=0.1))\n",
    "    b3 = tf.Variable(tf.constant(1.0, shape = [num_hidden]))\n",
    "\n",
    "    w4 = tf.Variable(tf.truncated_normal([num_hidden, num_hidden], stddev=0.1))\n",
    "    b4 = tf.Variable(tf.constant(1.0, shape = [num_hidden]))\n",
    "    \n",
    "    w5 = tf.Variable(tf.truncated_normal([num_hidden, num_labels], stddev=0.1))\n",
    "    b5 = tf.Variable(tf.constant(1.0, shape = [num_labels]))\n",
    "    variables = {\n",
    "        'w1': w1, 'w2': w2, 'w3': w3, 'w4': w4, 'w5': w5,\n",
    "        'b1': b1, 'b2': b2, 'b3': b3, 'b4': b4, 'b5': b5\n",
    "    }\n",
    "    return variables\n",
    "\n",
    "def model_lenet5_like(data, variables):\n",
    "    layer1_conv = tf.nn.conv2d(data, variables['w1'], [1, 1, 1, 1], padding='SAME')\n",
    "    layer1_actv = tf.nn.relu(layer1_conv + variables['b1'])\n",
    "    layer1_pool = tf.nn.avg_pool(layer1_actv, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "    layer2_conv = tf.nn.conv2d(layer1_pool, variables['w2'], [1, 1, 1, 1], padding='SAME')\n",
    "    layer2_actv = tf.nn.relu(layer2_conv + variables['b2'])\n",
    "    layer2_pool = tf.nn.avg_pool(layer2_actv, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')\n",
    "    \n",
    "    flat_layer = flatten_tf_array(layer2_pool)\n",
    "    layer3_fccd = tf.matmul(flat_layer, variables['w3']) + variables['b3']\n",
    "    layer3_actv = tf.nn.relu(layer3_fccd)\n",
    "    layer3_drop = tf.nn.dropout(layer3_actv, 0.5)\n",
    "    \n",
    "    layer4_fccd = tf.matmul(layer3_actv, variables['w4']) + variables['b4']\n",
    "    layer4_actv = tf.nn.relu(layer4_fccd)\n",
    "    layer4_drop = tf.nn.dropout(layer4_actv, 0.5)\n",
    "    \n",
    "    logits = tf.matmul(layer4_actv, variables['w5']) + variables['b5']\n",
    "    return logits\n",
    "\n",
    "\n",
    "#Variables used in the constructing and running the graph\n",
    "num_steps = 10001\n",
    "display_step = 1000\n",
    "learning_rate = 0.001\n",
    "batch_size = 16\n",
    "\n",
    "#定义数据的基本信息，传入变量\n",
    "image_width = 32\n",
    "image_height = 32\n",
    "image_depth = 3\n",
    "num_labels = 10\n",
    "\n",
    "\n",
    "test_dataset = test_dataset_cifar10\n",
    "test_labels = test_labels_cifar10\n",
    "train_dataset = train_dataset_cifar10\n",
    "train_labels = train_labels_cifar10\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    #1 首先使用占位符定义数据变量的维度\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_width, image_height, image_depth))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape = (batch_size, num_labels))\n",
    "    tf_test_dataset = tf.constant(test_dataset, tf.float32)\n",
    "\n",
    "    #2 然后初始化权重矩阵和偏置向量\n",
    "    variables = variables_lenet5_like(image_width = image_width, image_height=image_height, image_depth = image_depth, num_labels = num_labels)\n",
    "\n",
    "\n",
    "    #3 使用模型计算分类\n",
    "    logits = model_lenet5_like(tf_train_dataset, variables)\n",
    "\n",
    "    #4 使用带softmax的交叉熵函数计算预测标签和真实标签之间的损失函数\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=tf_train_labels))\n",
    "\n",
    "    #5  采用Adam优化算法优化上一步定义的损失函数，给定学习率\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "\n",
    "    # 执行预测推断\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    test_prediction = tf.nn.softmax(model_lenet5_like(tf_test_dataset, variables))\n",
    "\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    #初始化全部变量\n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Initialized with learning_rate', learning_rate)\n",
    "    for step in range(num_steps):\n",
    "        offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "        batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n",
    "        batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "        #在每一次批量中，获取当前的训练数据，并传入feed_dict以馈送到占位符中\n",
    "        feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "        train_accuracy = accuracy(predictions, batch_labels)\n",
    "        \n",
    "        if step % display_step == 0:\n",
    "            test_accuracy = accuracy(test_prediction.eval(), test_labels)\n",
    "            message = \"step {:04d} : loss is {:06.2f}, accuracy on training set {:02.2f} %, accuracy on test set {:02.2f} %\".format(step, l, train_accuracy, test_accuracy)\n",
    "            print(message)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
